Introduction

A key test of Milo is how it compare to other other methods. For this we need a ground truth that is vaguely realistic. I will use the linear trajectory simulation with 2000 cells where a single middle group of cells are differentially abundant between conditions. The methods against which I will make this comparison are:

To compare methods I will calculate the ratio of cells that fall into true positive DA regions/clusters/neighbourhoods to false positive DA regions/clusters/neighbourhoods.

### Set up a mock data set using simulated data
library(ggplot2)
library(igraph)
library(ggthemes)
library(ggsci)
library(umap)
library(reshape2)
library(SingleCellExperiment)
library(scran)
library(scater)
library(igraph)
library(miloR)
library(cowplot)
library(RColorBrewer)
library(pheatmap)
library(DAseq)
library(cydar)

I’ll use the simple linear trajectory data set for this with ~2000 cells and genuinely DA regions.

n.dim <- 15
k <- 10
sim.data <- readRDS("~/Dropbox/Milo/simulations/data/Trajectory_Ncells2000_3M1DARep100_simpleSim.RDS")
sim.mylo <- sim.data$mylo
sim.meta <- sim.data$meta
sim.mylo <- buildGraph(sim.mylo, k=k, d=n.dim, seed=42)
sim.mylo
class: Milo 
dim: 2000 2000 
metadata(0):
assays(2): counts logcounts
rownames(2000): G1 G2 ... G1999 G2000
rowData names(0):
colnames(2000): C1 C2 ... C1999 C2000
colData names(2): cell_id group_id
reducedDimNames(1): PCA
spikeNames(0):
altExpNames(0):
nhoods dimensions(1): 0
nhoodCounts dimensions(2): 1 1
nhoodDistances dimensions(2): 2000 2000
graph names(1): graph
nhoodIndex names(1): 0
nhoodExpression dimension(2): 1 1
nhoodReducedDim names(0):
nhoodGraph names(0):

I’ll create an embedding that can be used across all comparisons.

set.seed(42)
sim.graph <- miloR::graph(sim.mylo)
sim.fr_layout <- layout_with_fr(sim.graph)
sim.fr.df <- as.data.frame(sim.fr_layout)
sim.fr.df$cell_id <- colnames(sim.mylo)
sim.fr.df <- merge(sim.fr.df, sim.meta, by='cell_id')
rownames(sim.fr.df) <- sim.fr.df$cell_id
ggplot(sim.fr.df, aes(x=V1, y=V2)) +
    geom_point(aes(fill=Condition), size=3, shape=21) +
    scale_fill_manual(values=c("#662483", "white")) +
    theme_cowplot() +
    theme(axis.line=element_blank(), axis.ticks=element_blank(),
          axis.text=element_blank(), axis.title=element_blank()) +
    #facet_wrap(~Condition) +
    guides(fill=guide_legend(title="Condition", override.aes=list(size=3)),
           colour=FALSE, shape=FALSE, size=FALSE, alpha=FALSE) +
    #facet_wrap(~Condition, nrow=1) +
    NULL

ggsave("~/Dropbox/Milo/figures/MethodCompare_GroundTruth.png",
       height=4.15, width=8.25, dpi=300)
ggsave("~/Dropbox/Milo/figures/MethodCompare_GroundTruth.pdf",
       height=4.15, width=8.25, useDingbats=FALSE)
ggplot(sim.fr.df, aes(x=group_id, fill=Condition)) +
    geom_bar(position='dodge', colour='black') +
    scale_fill_manual(values=c("#662483", "white")) +
    theme_cowplot() +
    labs(x="Cell Group", y="#Cells") +
    NULL

ggsave("~/Dropbox/Milo/figures/MethodCompare_simulation_bar.pdf",
       height=2.15, width=3.15, useDingbats=FALSE)

Milo

set.seed(42)
sim.mylo <- buildGraph(sim.mylo, k=k, d=n.dim, seed=42)
Constructing kNN graph with k:10
Retrieving distances from 10 nearest neighbours
test.meta <- data.frame("Condition"=c(rep("A", 3), rep("B", 3)),
                        "Replicate"=rep(c("R1", "R2", "R3"), 2))
test.meta$Sample <- paste(test.meta$Condition, test.meta$Replicate, sep="_")
rownames(test.meta) <- test.meta$Sample
sim.mylo <- makeNhoods(sim.mylo, k=k, d=n.dim, prop=0.3, refined=TRUE)
Checking valid object
sim.mylo <- miloR::countCells(sim.mylo, samples="Sample", meta.data=as.data.frame(sim.meta))
Checking meta.data validity
Setting up matrix with 311 neighbourhoods
Counting cells in neighbourhoods
mylo.res <- testNhoods(sim.mylo, design=~Condition, design.df=test.meta[colnames(nhoodCounts(sim.mylo)), ])
Performing spatial FDR correction withk-distance weightingPerforming spatial FDR correction withneighbour-distance weightingPerforming spatial FDR correction withedge weightingPerforming spatial FDR correction withvertex weightingPerforming spatial FDR correction withnone weighting
mylo.res$Diff <- sign(mylo.res$logFC)
mylo.res$Diff[mylo.res$SpatialFDR > 0.1] <- 0
table(mylo.res$Diff)

  0   1 
252  59 

Cydar

Cydar requires the user to define a space in which to construct hyperspheres of a specific radius \(r\). I will use the same number of PCs as was used to construct the kNN-graph with Milo; \(r\) will have to be set by some other means.

sim.list <- list()
for(x in seq_along(unique(sim.meta$Replicate))){
  plate <- unique(sim.meta$Replicate)[x]
  plate.red <- sim.meta[sim.meta$Replicate == plate, ]
  plate.ages <- unique(plate.red$Condition)
  for(i in seq_along(plate.ages)){
    age <- unique(plate.ages)[i]
    age.red <- reducedDim(sim.mylo)[sim.meta$Condition == age &
                                        sim.meta$Replicate %in% plate, ]
    
    age.mat <- as(age.red[, 1:n.dim], "matrix")
    sim.list[[paste(age, paste0(plate), sep=".")]] <- age.mat
  }
}
sim.cydar <- prepareCellData(sim.list)

The key paramater for Cydar is the radius of the hyperspheres - this can be selected heuristically by plotting the distribution of distances for increasing values of \(r\).

This looks like the distances plateau after ~1.5 We can then count cells in hyperspheres and perform DA testing using edgeR.

   Mode   FALSE    TRUE 
logical     400     218 

Cydar finds 363 DA hyperspheres in this example.

DAseq

DAseq requires a range of k-values to be input, I’ll vary from 5 up to 50. NB: Should this actually be a set of values that are more realistic for the method?

# k.vec <- c(5, 7, 10, 12, 15, 20, 25, 30, 35, 40, 45, 50)
k.vec <- c(5, 500, 50)
sim.daseq <- getDAcells(X=reducedDim(sim.mylo)[, 1:n.dim],
                        cell.labels=sim.meta$cell_id,
                        labels.1=sim.meta$cell_id[sim.meta$Condition %in% c("A")],
                        labels.2=sim.meta$cell_id[sim.meta$Condition %in% c("B")],
                        k.vector=k.vec,
                        size=1,
                        plot.embedding=as.matrix(sim.fr.df[, c("V1", "V2")]))
Calculating DA score vector.
Running GLM.

Let’s have a look at these regions.

str(sim.daseq[1:4])
List of 4
 $ da.ratio: num [1:2000, 1:3] -0.6945 0.0393 -0.3506 0.4852 -0.3506 ...
  ..- attr(*, "dimnames")=List of 2
  .. ..$ : NULL
  .. ..$ : chr [1:3] "5" "500" "50"
 $ da.pred : num [1:2000] 0.278 0.578 0.394 0.831 0.432 ...
 $ da.up   : int [1:100] 79 86 94 111 130 133 134 188 206 223 ...
 $ da.down : int [1:100] 10 30 58 84 85 129 146 186 201 215 ...
sim.daseq$pred.plot

This plot shows what DAseq predicts as being as the DA cells, i.e. different between conditions A and B. By default the DA cells are selected in the top and bottom 5% of quantiles - I’ll keep this as it will select the best 10% overall.

sim.daseq$da.cells.plot

These top 10% of DA cells are specifically highlighted here. The DA regions are identified by grouping the coherently DA cells together by DAseq.

sim.da_regions <- getDAregion(X=reducedDim(sim.mylo)[, 1:n.dim],
                              da.cells=sim.daseq,
                              min.cell=5,
                              cell.labels=sim.meta$cell_id,
                              labels.1=sim.meta$cell_id[sim.meta$Condition %in% c("A")],
                              labels.2=sim.meta$cell_id[sim.meta$Condition %in% c("B")],
                              size=1,
                              resolution=0.1, plot.embedding=as.matrix(sim.fr.df[, c("V1", "V2")]))
Removing 1 DA regions with cells < 5.
str(sim.da_regions[1:2])
List of 2
 $ da.region.label: num [1:2000] 0 0 0 0 0 0 0 0 0 2 ...
 $ DA.stat        : num [1:5, 1:3] 1 -0.634 -0.748 -0.93 -1 ...
  ..- attr(*, "dimnames")=List of 2
  .. ..$ : NULL
  .. ..$ : chr [1:3] "DA.score" "pval.wilcoxon" "pval.ttest"

DAseq has clustered the DA cells into 6 DA regions.

sim.da_regions$da.region.plot

Clustering - Louvain & Walktrap

walktrap.clust <- cluster_walktrap(sim.graph, steps=3, membership=TRUE)
walktrap.clust.ids <- membership(walktrap.clust)
louvain.clust <- cluster_louvain(sim.graph)
louvain.clust.ids <- membership(louvain.clust)
sim.clust.df <- data.frame("cell_id"=colnames(sim.mylo), "Walktrap.Clust"=as.character(walktrap.clust.ids),
                           "Louvain.Clust"=as.character(louvain.clust.ids))
sim.clust.merge <- merge(sim.fr.df, sim.clust.df, by='cell_id')
sim.clust.merge$Sample <- paste(sim.fr.df$Condition, sim.fr.df$Replicate, sep="_")

Louvain

louvain.model <- model.matrix(~Condition, data=test.meta)
louvain.count <- as.matrix(table(sim.clust.merge$Louvain.Clust, sim.clust.merge$Sample))
louvain.dge <- DGEList(counts=louvain.count, lib.size=log(colSums(louvain.count)))
louvain.dge <- estimateDisp(louvain.dge, louvain.model)
louvain.fit <- glmQLFit(louvain.dge, louvain.model, robust=TRUE)
louvain.res <- as.data.frame(topTags(glmQLFTest(louvain.fit, coef=2), sort.by='none', n=Inf))
table(louvain.res$FDR <= 0.1)

FALSE  TRUE 
    7     2 

Walktrap

walktrap.model <- model.matrix(~Condition, data=test.meta)
walktrap.count <- as.matrix(table(sim.clust.merge$Walktrap.Clust, sim.clust.merge$Sample))
walktrap.dge <- DGEList(counts=walktrap.count, lib.size=log(colSums(walktrap.count)))
walktrap.dge <- estimateDisp(walktrap.dge, walktrap.model)
walktrap.fit <- glmQLFit(walktrap.dge, walktrap.model, robust=TRUE)
walktrap.res <- as.data.frame(topTags(glmQLFTest(walktrap.fit, coef=2), sort.by='none', n=Inf))
table(walktrap.res$FDR <= 0.1)

FALSE  TRUE 
    8     2 

Comparing methods

As stated above, I will assess the performance of each methods by calculating the ratio of cells that should be DA against those that are DA but should not be.

true.da.cells <- sim.meta$cell_id[sim.meta$group_id %in% "M3"]
milo.da.cells <- unique(sim.meta$cell_id[unique(unlist(nhoods(sim.mylo)[mylo.res$Nhood[mylo.res$Diff != 0]]))])
cydar.da.cells <- sim.meta$cell_id[unique(unlist(cellAssignments(sim.cydar)[as.numeric(rownames(cydar.res)[cydar.res$SpatialFDR <= 0.1])]))]
louvain.da.cells <- sim.clust.merge$cell_id[sim.clust.merge$Louvain.Clust %in% rownames(louvain.res)[louvain.res$FDR <= 0.1]]
walktrap.da.cells <- sim.clust.merge$cell_id[sim.clust.merge$Walktrap.Clust %in% rownames(walktrap.res)[walktrap.res$FDR <= 0.1]]
daseq.da.cells <- sim.meta$cell_id[sim.da_regions$da.region.label != 0]
da.cell.df <- data.frame("Method"=c("Truth", "Milo", "Cydar", "Louvain", "Walktrap", "DAseq"),
                         "NCells"=c(length(true.da.cells), length(milo.da.cells), length(cydar.da.cells),
                                    length(louvain.da.cells), length(walktrap.da.cells), length(daseq.da.cells)))
ggplot(da.cell.df, aes(x=reorder(Method, -NCells), y=NCells)) +
    geom_bar(stat='identity') +
    theme_clean() +
    labs(x="DA Method", y="#DA cells")
ggsave("~/Dropbox/Milo/figures/MethodCompare_NDAcells.pdf",
       height=3.95, width=4.25, useDingbats=FALSE)

From this it does look like Milo will call some neighbourhoods that are DA because the ratio of cells in that neighbourhood departs from 50:50. I’m not entirely sure how sensitive Milo is to this departure exactly, but it looks like it calls ~80 or so cells DA where they should not be.

# calculate true DA cells
true.neg <- factor(sim.meta$cell_id %in% true.da.cells, levels=c(FALSE, TRUE))
milo.neg <- factor(sim.meta$cell_id %in% milo.da.cells, levels=c(FALSE, TRUE))
cydar.neg <- factor(sim.meta$cell_id %in% cydar.da.cells, levels=c(FALSE, TRUE))
louvain.neg <- factor(sim.meta$cell_id %in% louvain.da.cells, levels=c(FALSE, TRUE))
walktrap.neg <- factor(sim.meta$cell_id %in% walktrap.da.cells, levels=c(FALSE, TRUE))
daseq.neg <- factor(sim.meta$cell_id %in% daseq.da.cells, levels=c(FALSE, TRUE))
milo.confuse <- table(true.neg, milo.neg)
cydar.confuse <- table(true.neg, cydar.neg)
louvain.confuse <- table(true.neg, louvain.neg)
walktrap.confuse <- table(true.neg, walktrap.neg)
daseq.confuse <- table(true.neg, daseq.neg)
milo.confuse.list <- list("TP"=milo.confuse[2, 2], "FP"=milo.confuse[1, 2], "TN"=milo.confuse[1, 1], "FN"=milo.confuse[2, 1])
cydar.confuse.list <- list("TP"=cydar.confuse[2, 2], "FP"=cydar.confuse[1, 2], "TN"=cydar.confuse[1, 1], "FN"=cydar.confuse[2, 1])
louvain.confuse.list <- list("TP"=louvain.confuse[2, 2], "FP"=louvain.confuse[1, 2], "TN"=louvain.confuse[1, 1], "FN"=louvain.confuse[2, 1])
walktrap.confuse.list <- list("TP"=walktrap.confuse[2, 2], "FP"=walktrap.confuse[1, 2], "TN"=walktrap.confuse[1, 1], "FN"=walktrap.confuse[2, 1])
daseq.confuse.list <- list("TP"=daseq.confuse[2, 2], "FP"=daseq.confuse[1, 2], "TN"=daseq.confuse[1, 1], "FN"=daseq.confuse[2, 1])
milo.ppv <- milo.confuse.list$TP/(milo.confuse.list$TP + milo.confuse.list$FP)
cydar.ppv <- cydar.confuse.list$TP/(cydar.confuse.list$TP + cydar.confuse.list$FP)
louvain.ppv <- louvain.confuse.list$TP/(louvain.confuse.list$TP + louvain.confuse.list$FP)
walktrap.ppv <- walktrap.confuse.list$TP/(walktrap.confuse.list$TP + walktrap.confuse.list$FP)
daseq.ppv <- daseq.confuse.list$TP/(daseq.confuse.list$TP + daseq.confuse.list$FP)
milo.fdr <- milo.confuse.list$FP/(milo.confuse.list$TP + milo.confuse.list$FP)
cydar.fdr <- cydar.confuse.list$FP/(cydar.confuse.list$TP + cydar.confuse.list$FP)
louvain.fdr <- louvain.confuse.list$FP/(louvain.confuse.list$TP + louvain.confuse.list$FP)
walktrap.fdr <- walktrap.confuse.list$FP/(walktrap.confuse.list$TP + walktrap.confuse.list$FP)
daseq.fdr <- daseq.confuse.list$FP/(daseq.confuse.list$TP + daseq.confuse.list$FP)
milo.fnr <- milo.confuse.list$FN/(milo.confuse.list$TP + milo.confuse.list$FN)
cydar.fnr <- cydar.confuse.list$FN/(cydar.confuse.list$TP + cydar.confuse.list$FN)
louvain.fnr <- louvain.confuse.list$FN/(louvain.confuse.list$TP + louvain.confuse.list$FN)
walktrap.fnr <- walktrap.confuse.list$FN/(walktrap.confuse.list$TP + walktrap.confuse.list$FN)
daseq.fnr <- daseq.confuse.list$FN/(daseq.confuse.list$TP + daseq.confuse.list$FN)
milo.fpr <- milo.confuse.list$FP/(milo.confuse.list$FP + milo.confuse.list$TN)
cydar.fpr <- cydar.confuse.list$FP/(cydar.confuse.list$FP + cydar.confuse.list$TN)
louvain.fpr <- louvain.confuse.list$FP/(louvain.confuse.list$FP + louvain.confuse.list$TN)
walktrap.fpr <- walktrap.confuse.list$FP/(walktrap.confuse.list$FP + walktrap.confuse.list$TN)
daseq.fpr <- daseq.confuse.list$FP/(daseq.confuse.list$FP + daseq.confuse.list$TN)
milo.for <- milo.confuse.list$FN/(milo.confuse.list$FN + milo.confuse.list$TN)
cydar.for <- cydar.confuse.list$FN/(cydar.confuse.list$FN + cydar.confuse.list$TN)
louvain.for <- louvain.confuse.list$FN/(louvain.confuse.list$FN + louvain.confuse.list$TN)
walktrap.for <- walktrap.confuse.list$FN/(walktrap.confuse.list$FN + walktrap.confuse.list$TN)
daseq.for <- daseq.confuse.list$FN/(daseq.confuse.list$FN + daseq.confuse.list$TN)
milo.tpr <- milo.confuse.list$TP/(milo.confuse.list$TP + milo.confuse.list$FN)
cydar.tpr <- cydar.confuse.list$TP/(cydar.confuse.list$TP + cydar.confuse.list$FN)
louvain.tpr <- louvain.confuse.list$TP/(louvain.confuse.list$TP + louvain.confuse.list$FN)
walktrap.tpr <- walktrap.confuse.list$TP/(walktrap.confuse.list$TP + walktrap.confuse.list$FN)
daseq.tpr <- daseq.confuse.list$TP/(daseq.confuse.list$TP + daseq.confuse.list$FN)
milo.tnr <- milo.confuse.list$TN/(milo.confuse.list$TN + milo.confuse.list$FP)
cydar.tnr <- cydar.confuse.list$TN/(cydar.confuse.list$TN + cydar.confuse.list$FP)
louvain.tnr <- louvain.confuse.list$TN/(louvain.confuse.list$TN + louvain.confuse.list$FP)
walktrap.tnr <- walktrap.confuse.list$TN/(walktrap.confuse.list$TN + walktrap.confuse.list$FP)
daseq.tnr <- daseq.confuse.list$TN/(daseq.confuse.list$TN + daseq.confuse.list$FP)
milo.npv <- milo.confuse.list$TN/(milo.confuse.list$TN + milo.confuse.list$FN)
cydar.npv <- cydar.confuse.list$TN/(cydar.confuse.list$TN + cydar.confuse.list$FN)
louvain.npv <- louvain.confuse.list$TN/(louvain.confuse.list$TN + louvain.confuse.list$FN)
walktrap.npv <- walktrap.confuse.list$TN/(walktrap.confuse.list$TN + walktrap.confuse.list$FN)
daseq.npv <- daseq.confuse.list$TN/(daseq.confuse.list$TN + daseq.confuse.list$FN)
da.fdr.df <- data.frame("Method"=c("Milo", "Cydar", "Louvain", "Walktrap", "DAseq"),
                        "PPV"=c(milo.ppv, cydar.ppv, louvain.ppv, walktrap.ppv, daseq.ppv),
                        "NPV"=c(milo.npv, cydar.npv, louvain.npv, walktrap.npv, daseq.npv),
                        "FDR"=c(milo.fdr, cydar.fdr, louvain.fdr, walktrap.fdr, daseq.fdr),
                        "FPR"=c(milo.fpr, cydar.fpr, louvain.fpr, walktrap.fpr, daseq.fpr),
                        "FNR"=c(milo.fnr, cydar.fnr, louvain.fnr, walktrap.fnr, daseq.fnr),
                        "FOR"=c(milo.for, cydar.for, louvain.for, walktrap.for, daseq.for),
                        "TPR"=c(milo.tpr, cydar.tpr, louvain.tpr, walktrap.tpr, daseq.tpr),
                        "TNR"=c(milo.tnr, cydar.tnr, louvain.tnr, walktrap.tnr, daseq.tnr))
da.fdr.df$MCC <- sqrt(da.fdr.df$PPV * da.fdr.df$TPR * da.fdr.df$TNR * da.fdr.df$NPV) - 
    sqrt(da.fdr.df$FDR * da.fdr.df$FNR * da.fdr.df$FPR * da.fdr.df$FOR)
da.fdr.df$F1 <- 2*((da.fdr.df$PPV * da.fdr.df$TPR)/(da.fdr.df$PPV + da.fdr.df$TPR))
da.fdr.df$Power <- 1 - da.fdr.df$FNR

I’ve compute the confusion matrix for each method, from which I have calculated the precision (positive predictive value), recall (TPR), FDR and Matthews correlation coefficient (MCC).

ggplot(da.fdr.df, aes(x=reorder(Method, -PPV), y=PPV)) +
    geom_bar(stat='identity') +
    theme_clean() +
    labs(x="DA Method", y="PPV")

ggsave("~/Dropbox/Milo/figures/MethodCompare_PPV.pdf",
       height=3.95, width=4.25, useDingbats=FALSE)
ggplot(da.fdr.df, aes(x=reorder(Method, -FDR), y=FDR)) +
    geom_bar(stat='identity') +
    geom_hline(yintercept=0.1, lty=2, col='red') +
    theme_clean() +
    labs(x="DA Method", y="Cell-wise FDR")

ggsave("~/Dropbox/Milo/figures/MethodCompare_FDR.pdf",
       height=3.95, width=4.25, useDingbats=FALSE)
ggplot(da.fdr.df, aes(x=reorder(Method, -TPR), y=TPR)) +
    geom_bar(stat='identity') +
    theme_clean() +
    labs(x="DA Method", y="Recall")

ggsave("~/Dropbox/Milo/figures/MethodCompare_Recall.pdf",
       height=3.95, width=4.25, useDingbats=FALSE)
ggplot(da.fdr.df, aes(x=reorder(Method, -MCC), y=MCC)) +
    geom_bar(stat='identity') +
    theme_cowplot() +
    theme(axis.text=element_text(size=20),
          axis.title=element_text(size=22)) +
    labs(x="Method", y="MCC")

ggsave("~/Dropbox/Milo/figures/MethodCompare_MCC.pdf",
       height=2.95, width=6.95, useDingbats=FALSE)

What does the power look like?

ggplot(da.fdr.df, aes(x=reorder(Method, -Power), y=Power)) +
    geom_bar(stat='identity') +
    theme_clean() +
    labs(x="DA Method", y="Power")

ggsave("~/Dropbox/Milo/figures/MethodCompare_Power.pdf",
       height=3.95, width=4.25, useDingbats=FALSE)

Let’s visualise these in a single table/matrix. I have to split this into the measures where higher is better and lower is better.

# calculate the rank along each column
da.negrank.df <- as.data.frame(apply(da.fdr.df[, c("Power", "F1", "TNR", "TPR", "NPV", "PPV")],
                                     2, FUN=function(X) rank(-X)))
da.negrank.df$Method <- da.fdr.df$Method
da.negrank.melt <- melt(da.negrank.df, id.vars=c("Method"))
da.negrank.melt$value <- ordered(da.negrank.melt$value,
                                 levels=c(1:5))
rank.cols <- colorRampPalette(pal_futurama()(3))(5)
names(rank.cols) <- c(1:5)
ggplot(da.negrank.melt, 
       aes(x=Method, y=variable, fill=value)) +
    geom_tile() +
    theme_cowplot() +
    scale_fill_manual(values=rank.cols) +
    labs(x="Method", y="Measure") +
     theme(axis.text=element_text(size=18),
          axis.title=element_text(size=22),
          legend.text=element_text(size=18),
          legend.title=element_text(size=20)) +
    guides(fill=guide_legend(title="Rank"))
ggsave("~/Dropbox/Milo/figures/MethodCompare_PosRank_table.pdf",
       height=3.95, width=6.25, useDingbats=FALSE)

# calculate the rank along each column
da.posrank.df <- as.data.frame(apply(da.fdr.df[, c("FDR", "FPR", "FNR", "FOR")],
                                     2, FUN=function(X) rank(X)))
da.posrank.df$Method <- da.fdr.df$Method
da.posrank.melt <- melt(da.posrank.df, id.vars=c("Method"))
da.posrank.melt$value <- ordered(da.posrank.melt$value,
                                 levels=c(1:5))
rank.cols <- colorRampPalette(pal_futurama()(3))(5)
names(rank.cols) <- c(1:5)
ggplot(da.posrank.melt, 
       aes(x=Method, y=variable, fill=value)) +
    geom_tile() +
    theme_cowplot() +
    scale_fill_manual(values=rank.cols) +
    labs(x="Method", y="Measure") +
    theme(axis.text=element_text(size=18),
          axis.title=element_text(size=22),
          legend.text=element_text(size=18),
          legend.title=element_text(size=20)) +
    guides(fill=guide_legend(title="Rank"))
ggsave("~/Dropbox/Milo/figures/MethodCompare_NegRank_table.pdf",
       height=3.95, width=6.25, useDingbats=FALSE)

# calculate the rank along each column
allrank.melt <- do.call(rbind.data.frame, list("post"=da.posrank.melt, "neg"=da.negrank.melt))
rank.cols <- colorRampPalette(pal_futurama()(3))(5)
names(rank.cols) <- c(1:5)
ggplot(allrank.melt, 
       aes(x=Method, y=variable, fill=value)) +
    geom_tile() +
    theme_cowplot() +
    scale_fill_manual(values=rank.cols) +
    labs(x="Method", y="Measure") +
     theme(axis.text=element_text(size=18),
          axis.title=element_text(size=22),
          legend.text=element_text(size=18),
          legend.title=element_text(size=20)) +
    guides(fill=guide_legend(title="Rank"))
ggsave("~/Dropbox/Milo/figures/MethodCompare_AllRank_table.pdf",
       height=4.15, width=6.95, useDingbats=FALSE)

LS0tCnRpdGxlOiAiTWlsbzogY29tcGFyaXNvbiB0byBvdGhlciBtZXRob2RzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIEludHJvZHVjdGlvbgoKQSBrZXkgdGVzdCBvZiBgTWlsb2AgaXMgaG93IGl0IGNvbXBhcmUgdG8gb3RoZXIgb3RoZXIgbWV0aG9kcy4gRm9yIHRoaXMgd2UgbmVlZCBhIGdyb3VuZCB0cnV0aCB0aGF0IGlzIHZhZ3VlbHkgcmVhbGlzdGljLiBJIHdpbGwgdXNlIHRoZSBsaW5lYXIgCnRyYWplY3Rvcnkgc2ltdWxhdGlvbiB3aXRoIDIwMDAgY2VsbHMgd2hlcmUgYSBzaW5nbGUgbWlkZGxlIGdyb3VwIG9mIGNlbGxzIGFyZSBkaWZmZXJlbnRpYWxseSBhYnVuZGFudCBiZXR3ZWVuIGNvbmRpdGlvbnMuICBUaGUgbWV0aG9kcyBhZ2FpbnN0IHdoaWNoIApJIHdpbGwgbWFrZSB0aGlzIGNvbXBhcmlzb24gYXJlOgoKKiBDbHVzdGVyLWJhc2VkOiBMb3V2YWluIGFuZCBXYWxrdHJhcAoqIEN5ZGFyCiogREEtc2VxCgpUbyBjb21wYXJlIG1ldGhvZHMgSSB3aWxsIGNhbGN1bGF0ZSB0aGUgcmF0aW8gb2YgY2VsbHMgdGhhdCBmYWxsIGludG8gdHJ1ZSBwb3NpdGl2ZSBEQSByZWdpb25zL2NsdXN0ZXJzL25laWdoYm91cmhvb2RzIHRvIGZhbHNlIHBvc2l0aXZlIERBIApyZWdpb25zL2NsdXN0ZXJzL25laWdoYm91cmhvb2RzLgoKYGBge3IsIGVjaG89VFJVRSwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KIyMjIFNldCB1cCBhIG1vY2sgZGF0YSBzZXQgdXNpbmcgc2ltdWxhdGVkIGRhdGEKbGlicmFyeShnZ3Bsb3QyKQpsaWJyYXJ5KGlncmFwaCkKbGlicmFyeShnZ3RoZW1lcykKbGlicmFyeShnZ3NjaSkKbGlicmFyeSh1bWFwKQpsaWJyYXJ5KHJlc2hhcGUyKQpsaWJyYXJ5KFNpbmdsZUNlbGxFeHBlcmltZW50KQpsaWJyYXJ5KHNjcmFuKQpsaWJyYXJ5KHNjYXRlcikKbGlicmFyeShpZ3JhcGgpCmxpYnJhcnkobWlsb1IpCmxpYnJhcnkoY293cGxvdCkKbGlicmFyeShSQ29sb3JCcmV3ZXIpCmxpYnJhcnkocGhlYXRtYXApCmxpYnJhcnkoREFzZXEpCmxpYnJhcnkoY3lkYXIpCmBgYAoKSSdsbCB1c2UgdGhlIHNpbXBsZSBsaW5lYXIgdHJhamVjdG9yeSBkYXRhIHNldCBmb3IgdGhpcyB3aXRoIH4yMDAwIGNlbGxzIGFuZCBnZW51aW5lbHkgREEgcmVnaW9ucy4KCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQpuLmRpbSA8LSAxNQprIDwtIDEwCnNpbS5kYXRhIDwtIHJlYWRSRFMoIn4vRHJvcGJveC9NaWxvL3NpbXVsYXRpb25zL2RhdGEvVHJhamVjdG9yeV9OY2VsbHMyMDAwXzNNMURBUmVwMTAwX3NpbXBsZVNpbS5SRFMiKQpzaW0ubXlsbyA8LSBzaW0uZGF0YSRteWxvCnNpbS5tZXRhIDwtIHNpbS5kYXRhJG1ldGEKc2ltLm15bG8gPC0gYnVpbGRHcmFwaChzaW0ubXlsbywgaz1rLCBkPW4uZGltLCBzZWVkPTQyKQpzaW0ubXlsbwpgYGAKCkknbGwgY3JlYXRlIGFuIGVtYmVkZGluZyB0aGF0IGNhbiBiZSB1c2VkIGFjcm9zcyBhbGwgY29tcGFyaXNvbnMuCgpgYGB7ciwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRSwgZmlnLmhlaWdodD00LjE1LCBmaWcud2lkdGg9OC4yNX0Kc2V0LnNlZWQoNDIpCnNpbS5ncmFwaCA8LSBtaWxvUjo6Z3JhcGgoc2ltLm15bG8pCnNpbS5mcl9sYXlvdXQgPC0gbGF5b3V0X3dpdGhfZnIoc2ltLmdyYXBoKQoKc2ltLmZyLmRmIDwtIGFzLmRhdGEuZnJhbWUoc2ltLmZyX2xheW91dCkKc2ltLmZyLmRmJGNlbGxfaWQgPC0gY29sbmFtZXMoc2ltLm15bG8pCnNpbS5mci5kZiA8LSBtZXJnZShzaW0uZnIuZGYsIHNpbS5tZXRhLCBieT0nY2VsbF9pZCcpCnJvd25hbWVzKHNpbS5mci5kZikgPC0gc2ltLmZyLmRmJGNlbGxfaWQKCmdncGxvdChzaW0uZnIuZGYsIGFlcyh4PVYxLCB5PVYyKSkgKwogICAgZ2VvbV9wb2ludChhZXMoZmlsbD1Db25kaXRpb24pLCBzaXplPTMsIHNoYXBlPTIxKSArCiAgICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXM9YygiIzY2MjQ4MyIsICJ3aGl0ZSIpKSArCiAgICB0aGVtZV9jb3dwbG90KCkgKwogICAgdGhlbWUoYXhpcy5saW5lPWVsZW1lbnRfYmxhbmsoKSwgYXhpcy50aWNrcz1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRleHQ9ZWxlbWVudF9ibGFuaygpLCBheGlzLnRpdGxlPWVsZW1lbnRfYmxhbmsoKSkgKwogICAgI2ZhY2V0X3dyYXAofkNvbmRpdGlvbikgKwogICAgZ3VpZGVzKGZpbGw9Z3VpZGVfbGVnZW5kKHRpdGxlPSJDb25kaXRpb24iLCBvdmVycmlkZS5hZXM9bGlzdChzaXplPTMpKSwKICAgICAgICAgICBjb2xvdXI9RkFMU0UsIHNoYXBlPUZBTFNFLCBzaXplPUZBTFNFLCBhbHBoYT1GQUxTRSkgKwogICAgI2ZhY2V0X3dyYXAofkNvbmRpdGlvbiwgbnJvdz0xKSArCiAgICBOVUxMCgpnZ3NhdmUoIn4vRHJvcGJveC9NaWxvL2ZpZ3VyZXMvTWV0aG9kQ29tcGFyZV9Hcm91bmRUcnV0aC5wbmciLAogICAgICAgaGVpZ2h0PTQuMTUsIHdpZHRoPTguMjUsIGRwaT0zMDApCgpnZ3NhdmUoIn4vRHJvcGJveC9NaWxvL2ZpZ3VyZXMvTWV0aG9kQ29tcGFyZV9Hcm91bmRUcnV0aC5wZGYiLAogICAgICAgaGVpZ2h0PTQuMTUsIHdpZHRoPTguMjUsIHVzZURpbmdiYXRzPUZBTFNFKQpgYGAKCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFLCBmaWcuaGVpZ2h0PTIuMTUsIGZpZy53aWR0aD0zLjE1fQpnZ3Bsb3Qoc2ltLmZyLmRmLCBhZXMoeD1ncm91cF9pZCwgZmlsbD1Db25kaXRpb24pKSArCiAgICBnZW9tX2Jhcihwb3NpdGlvbj0nZG9kZ2UnLCBjb2xvdXI9J2JsYWNrJykgKwogICAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzPWMoIiM2NjI0ODMiLCAid2hpdGUiKSkgKwogICAgdGhlbWVfY293cGxvdCgpICsKICAgIGxhYnMoeD0iQ2VsbCBHcm91cCIsIHk9IiNDZWxscyIpICsKICAgIE5VTEwKCmdnc2F2ZSgifi9Ecm9wYm94L01pbG8vZmlndXJlcy9NZXRob2RDb21wYXJlX3NpbXVsYXRpb25fYmFyLnBkZiIsCiAgICAgICBoZWlnaHQ9Mi4xNSwgd2lkdGg9My4xNSwgdXNlRGluZ2JhdHM9RkFMU0UpCmBgYAoKCgojIE1pbG8KCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQpzZXQuc2VlZCg0MikKc2ltLm15bG8gPC0gYnVpbGRHcmFwaChzaW0ubXlsbywgaz1rLCBkPW4uZGltLCBzZWVkPTQyKQp0ZXN0Lm1ldGEgPC0gZGF0YS5mcmFtZSgiQ29uZGl0aW9uIj1jKHJlcCgiQSIsIDMpLCByZXAoIkIiLCAzKSksCiAgICAgICAgICAgICAgICAgICAgICAgICJSZXBsaWNhdGUiPXJlcChjKCJSMSIsICJSMiIsICJSMyIpLCAyKSkKdGVzdC5tZXRhJFNhbXBsZSA8LSBwYXN0ZSh0ZXN0Lm1ldGEkQ29uZGl0aW9uLCB0ZXN0Lm1ldGEkUmVwbGljYXRlLCBzZXA9Il8iKQpyb3duYW1lcyh0ZXN0Lm1ldGEpIDwtIHRlc3QubWV0YSRTYW1wbGUKCnNpbS5teWxvIDwtIG1ha2VOaG9vZHMoc2ltLm15bG8sIGs9aywgZD1uLmRpbSwgcHJvcD0wLjMsIHJlZmluZWQ9VFJVRSkKc2ltLm15bG8gPC0gbWlsb1I6OmNvdW50Q2VsbHMoc2ltLm15bG8sIHNhbXBsZXM9IlNhbXBsZSIsIG1ldGEuZGF0YT1hcy5kYXRhLmZyYW1lKHNpbS5tZXRhKSkKbXlsby5yZXMgPC0gdGVzdE5ob29kcyhzaW0ubXlsbywgZGVzaWduPX5Db25kaXRpb24sIGRlc2lnbi5kZj10ZXN0Lm1ldGFbY29sbmFtZXMobmhvb2RDb3VudHMoc2ltLm15bG8pKSwgXSkKbXlsby5yZXMkRGlmZiA8LSBzaWduKG15bG8ucmVzJGxvZ0ZDKQpteWxvLnJlcyREaWZmW215bG8ucmVzJFNwYXRpYWxGRFIgPiAwLjFdIDwtIDAKdGFibGUobXlsby5yZXMkRGlmZikKYGBgCgojIEN5ZGFyCgpgQ3lkYXJgIHJlcXVpcmVzIHRoZSB1c2VyIHRvIGRlZmluZSBhIHNwYWNlIGluIHdoaWNoIHRvIGNvbnN0cnVjdCBoeXBlcnNwaGVyZXMgb2YgYSBzcGVjaWZpYyByYWRpdXMgJHIkLiBJIHdpbGwgdXNlIHRoZSBzYW1lIG51bWJlciBvZiBQQ3MgYXMgd2FzIAp1c2VkIHRvIGNvbnN0cnVjdCB0aGUga05OLWdyYXBoIHdpdGggYE1pbG9gOyAkciQgd2lsbCBoYXZlIHRvIGJlIHNldCBieSBzb21lIG90aGVyIG1lYW5zLgoKYGBge3IsIHdhcm5pbmc9RkFMU0V9CnNpbS5saXN0IDwtIGxpc3QoKQpmb3IoeCBpbiBzZXFfYWxvbmcodW5pcXVlKHNpbS5tZXRhJFJlcGxpY2F0ZSkpKXsKICBwbGF0ZSA8LSB1bmlxdWUoc2ltLm1ldGEkUmVwbGljYXRlKVt4XQogIHBsYXRlLnJlZCA8LSBzaW0ubWV0YVtzaW0ubWV0YSRSZXBsaWNhdGUgPT0gcGxhdGUsIF0KICBwbGF0ZS5hZ2VzIDwtIHVuaXF1ZShwbGF0ZS5yZWQkQ29uZGl0aW9uKQogIGZvcihpIGluIHNlcV9hbG9uZyhwbGF0ZS5hZ2VzKSl7CiAgICBhZ2UgPC0gdW5pcXVlKHBsYXRlLmFnZXMpW2ldCiAgICBhZ2UucmVkIDwtIHJlZHVjZWREaW0oc2ltLm15bG8pW3NpbS5tZXRhJENvbmRpdGlvbiA9PSBhZ2UgJgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2ltLm1ldGEkUmVwbGljYXRlICVpbiUgcGxhdGUsIF0KICAgIAogICAgYWdlLm1hdCA8LSBhcyhhZ2UucmVkWywgMTpuLmRpbV0sICJtYXRyaXgiKQogICAgc2ltLmxpc3RbW3Bhc3RlKGFnZSwgcGFzdGUwKHBsYXRlKSwgc2VwPSIuIildXSA8LSBhZ2UubWF0CiAgfQp9CgpzaW0uY3lkYXIgPC0gcHJlcGFyZUNlbGxEYXRhKHNpbS5saXN0KQpgYGAKClRoZSBrZXkgcGFyYW1hdGVyIGZvciBgQ3lkYXJgIGlzIHRoZSByYWRpdXMgb2YgdGhlIGh5cGVyc3BoZXJlcyAtIHRoaXMgY2FuIGJlIHNlbGVjdGVkIGhldXJpc3RpY2FsbHkgYnkgcGxvdHRpbmcgdGhlIGRpc3RyaWJ1dGlvbiBvZiBkaXN0YW5jZXMgZm9yIAppbmNyZWFzaW5nIHZhbHVlcyBvZiAkciQuCgpgYGB7ciwgZWNobz1GQUxTRSwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0Kc2ltLmRpc3QgPC0gbmVpZ2hib3JEaXN0YW5jZXMoc2ltLmN5ZGFyLCBuZWlnaGJvcnM9NzUsIGFzLnRvbD1UUlVFKQpib3hwbG90KHNpbS5kaXN0KQpgYGAKClRoaXMgbG9va3MgbGlrZSB0aGUgZGlzdGFuY2VzIHBsYXRlYXUgYWZ0ZXIgfjEuNSBXZSBjYW4gdGhlbiBjb3VudCBjZWxscyBpbiBoeXBlcnNwaGVyZXMgYW5kIHBlcmZvcm0gREEgdGVzdGluZyB1c2luZyBgZWRnZVJgLgoKYGBge3IsIGVjaG89RkFMU0UsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0V9CnNpbS5jeWRhciA8LSBjeWRhcjo6Y291bnRDZWxscyhzaW0uY3lkYXIsIHRvbD0yLjAsIGZpbHRlcj0wLCBkb3duc2FtcGxlPTMpCm1lc3NhZ2UocGFzdGUwKCJDcmVhdGVkICIsIG5yb3coc2ltLmN5ZGFyKSwgIiBoeXBlcnNwaGVyZXMiKSkKCiMgZG8gREEgdGVzdGluZyB3aXRoIGVkZ2VSCnNpbS5kZ2UgPC0gREdFTGlzdChhc3NheShzaW0uY3lkYXIpLCBsaWIuc2l6ZT1zaW0uY3lkYXIkdG90YWxzKQoKIyBmaWx0ZXIgbG93IGFidW5kYW5jZSBoeXBlcnNwaGVyZXMKa2VlcCA8LSBhdmVMb2dDUE0oc2ltLmRnZSkgPj0gYXZlTG9nQ1BNKDEsIG1lYW4oc2ltLmN5ZGFyJHRvdGFscykpCnNpbS5jeWRhciA8LSBzaW0uY3lkYXJba2VlcCxdCnNpbS5kZ2UgPC0gc2ltLmRnZVtrZWVwLF0KCnNpbS5kZXNpZ24gPC0gbW9kZWwubWF0cml4KH5Db25kaXRpb24sIGRhdGE9dGVzdC5tZXRhW2dzdWIoY29sbmFtZXMoc2ltLmN5ZGFyKSwgcGF0dGVybj0iXFwuIiwgcmVwbGFjZW1lbnQ9Il8iKSwgXSkKc2ltLmRnZSA8LSBlc3RpbWF0ZURpc3Aoc2ltLmRnZSwgc2ltLmRlc2lnbikKc2ltLmZpdCA8LSBnbG1RTEZpdChzaW0uZGdlLCBzaW0uZGVzaWduKQpzaW0ucmVzIDwtIGdsbVFMRlRlc3Qoc2ltLmZpdCwgY29lZj0yKQoKIyBjb250cm9sIHRoZSBzcGF0aWFsIEZEUgpjeWRhci5yZXMgPC0gc2ltLnJlcyR0YWJsZQpjeWRhci5yZXMkU3BhdGlhbEZEUiA8LSBzcGF0aWFsRkRSKGludGVuc2l0aWVzKHNpbS5jeWRhciksIHNpbS5yZXMkdGFibGUkUFZhbHVlKQppcy5zaWcgPC0gY3lkYXIucmVzJFNwYXRpYWxGRFIgPD0gMC4xCnN1bW1hcnkoaXMuc2lnKQpgYGAKCmBDeWRhcmAgZmluZHMgMzYzIERBIGh5cGVyc3BoZXJlcyBpbiB0aGlzIGV4YW1wbGUuCgojIERBc2VxCgpgREFzZXFgIHJlcXVpcmVzIGEgcmFuZ2Ugb2Ygay12YWx1ZXMgdG8gYmUgaW5wdXQsIEknbGwgdmFyeSBmcm9tIDUgdXAgdG8gNTAuIF9fTkJfXzogU2hvdWxkIHRoaXMgYWN0dWFsbHkgYmUgYSBzZXQgb2YgdmFsdWVzIHRoYXQgYXJlIG1vcmUgcmVhbGlzdGljIApmb3IgdGhlIG1ldGhvZD8KCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQojIGsudmVjIDwtIGMoNSwgNywgMTAsIDEyLCAxNSwgMjAsIDI1LCAzMCwgMzUsIDQwLCA0NSwgNTApCmsudmVjIDwtIGMoNSwgNTAwLCA1MCkKc2ltLmRhc2VxIDwtIGdldERBY2VsbHMoWD1yZWR1Y2VkRGltKHNpbS5teWxvKVssIDE6bi5kaW1dLAogICAgICAgICAgICAgICAgICAgICAgICBjZWxsLmxhYmVscz1zaW0ubWV0YSRjZWxsX2lkLAogICAgICAgICAgICAgICAgICAgICAgICBsYWJlbHMuMT1zaW0ubWV0YSRjZWxsX2lkW3NpbS5tZXRhJENvbmRpdGlvbiAlaW4lIGMoIkEiKV0sCiAgICAgICAgICAgICAgICAgICAgICAgIGxhYmVscy4yPXNpbS5tZXRhJGNlbGxfaWRbc2ltLm1ldGEkQ29uZGl0aW9uICVpbiUgYygiQiIpXSwKICAgICAgICAgICAgICAgICAgICAgICAgay52ZWN0b3I9ay52ZWMsCiAgICAgICAgICAgICAgICAgICAgICAgIHNpemU9MSwKICAgICAgICAgICAgICAgICAgICAgICAgcGxvdC5lbWJlZGRpbmc9YXMubWF0cml4KHNpbS5mci5kZlssIGMoIlYxIiwgIlYyIildKSkKYGBgCgpMZXQncyBoYXZlIGEgbG9vayBhdCB0aGVzZSByZWdpb25zLgoKYGBge3IsIHdhcm5pbmc9RkFMU0V9CnN0cihzaW0uZGFzZXFbMTo0XSkKYGBgCgpgYGB7ciwgd2FybmluZz1GQUxTRSwgZmlnLmhlaWdodD00LjE1LCBmaWcud2lkdGg9NS4xNX0Kc2ltLmRhc2VxJHByZWQucGxvdApgYGAKClRoaXMgcGxvdCBzaG93cyB3aGF0IGBEQXNlcWAgcHJlZGljdHMgYXMgYmVpbmcgYXMgdGhlIERBIGNlbGxzLCBpLmUuIGRpZmZlcmVudCBiZXR3ZWVuIGNvbmRpdGlvbnMgQSBhbmQgQi4gQnkgZGVmYXVsdCB0aGUgREEgY2VsbHMgYXJlIHNlbGVjdGVkIGluIAp0aGUgdG9wIGFuZCBib3R0b20gNSUgb2YgcXVhbnRpbGVzIC0gSSdsbCBrZWVwIHRoaXMgYXMgaXQgd2lsbCBzZWxlY3QgdGhlIGJlc3QgMTAlIG92ZXJhbGwuCgpgYGB7ciwgd2FybmluZz1GQUxTRSwgZmlnLmhlaWdodD00LjE1LCBmaWcud2lkdGg9NS4xNX0Kc2ltLmRhc2VxJGRhLmNlbGxzLnBsb3QKYGBgCgpUaGVzZSB0b3AgMTAlIG9mIERBIGNlbGxzIGFyZSBzcGVjaWZpY2FsbHkgaGlnaGxpZ2h0ZWQgaGVyZS4gVGhlIERBIHJlZ2lvbnMgYXJlIGlkZW50aWZpZWQgYnkgZ3JvdXBpbmcgdGhlIGNvaGVyZW50bHkgREEgY2VsbHMgdG9nZXRoZXIgYnkgYERBc2VxYC4KCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQpzaW0uZGFfcmVnaW9ucyA8LSBnZXREQXJlZ2lvbihYPXJlZHVjZWREaW0oc2ltLm15bG8pWywgMTpuLmRpbV0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhLmNlbGxzPXNpbS5kYXNlcSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWluLmNlbGw9NSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY2VsbC5sYWJlbHM9c2ltLm1ldGEkY2VsbF9pZCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFiZWxzLjE9c2ltLm1ldGEkY2VsbF9pZFtzaW0ubWV0YSRDb25kaXRpb24gJWluJSBjKCJBIildLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYWJlbHMuMj1zaW0ubWV0YSRjZWxsX2lkW3NpbS5tZXRhJENvbmRpdGlvbiAlaW4lIGMoIkIiKV0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNpemU9MSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVzb2x1dGlvbj0wLjEsIHBsb3QuZW1iZWRkaW5nPWFzLm1hdHJpeChzaW0uZnIuZGZbLCBjKCJWMSIsICJWMiIpXSkpCgpzdHIoc2ltLmRhX3JlZ2lvbnNbMToyXSkKYGBgCgpgREFzZXFgIGhhcyBjbHVzdGVyZWQgdGhlIERBIGNlbGxzIGludG8gYHIgbGVuZ3RoKHVuaXF1ZShzaW0uZGFfcmVnaW9ucyRkYS5yZWdpb24ubGFiZWwpKWAgREEgcmVnaW9ucy4KCmBgYHtyLCBmaWcuaGVpZ2h0PTQuMTUsIGZpZy53aWR0aD01LjE1fQpzaW0uZGFfcmVnaW9ucyRkYS5yZWdpb24ucGxvdApgYGAKCiMgQ2x1c3RlcmluZyAtIExvdXZhaW4gJiBXYWxrdHJhcAoKYGBge3J9CndhbGt0cmFwLmNsdXN0IDwtIGNsdXN0ZXJfd2Fsa3RyYXAoc2ltLmdyYXBoLCBzdGVwcz0zLCBtZW1iZXJzaGlwPVRSVUUpCndhbGt0cmFwLmNsdXN0LmlkcyA8LSBtZW1iZXJzaGlwKHdhbGt0cmFwLmNsdXN0KQoKbG91dmFpbi5jbHVzdCA8LSBjbHVzdGVyX2xvdXZhaW4oc2ltLmdyYXBoKQpsb3V2YWluLmNsdXN0LmlkcyA8LSBtZW1iZXJzaGlwKGxvdXZhaW4uY2x1c3QpCgpzaW0uY2x1c3QuZGYgPC0gZGF0YS5mcmFtZSgiY2VsbF9pZCI9Y29sbmFtZXMoc2ltLm15bG8pLCAiV2Fsa3RyYXAuQ2x1c3QiPWFzLmNoYXJhY3Rlcih3YWxrdHJhcC5jbHVzdC5pZHMpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAiTG91dmFpbi5DbHVzdCI9YXMuY2hhcmFjdGVyKGxvdXZhaW4uY2x1c3QuaWRzKSkKCnNpbS5jbHVzdC5tZXJnZSA8LSBtZXJnZShzaW0uZnIuZGYsIHNpbS5jbHVzdC5kZiwgYnk9J2NlbGxfaWQnKQpzaW0uY2x1c3QubWVyZ2UkU2FtcGxlIDwtIHBhc3RlKHNpbS5mci5kZiRDb25kaXRpb24sIHNpbS5mci5kZiRSZXBsaWNhdGUsIHNlcD0iXyIpCmBgYAoKIyMgTG91dmFpbgoKYGBge3IsIHdhcm5pbmc9RkFMU0V9CmxvdXZhaW4ubW9kZWwgPC0gbW9kZWwubWF0cml4KH5Db25kaXRpb24sIGRhdGE9dGVzdC5tZXRhKQpsb3V2YWluLmNvdW50IDwtIGFzLm1hdHJpeCh0YWJsZShzaW0uY2x1c3QubWVyZ2UkTG91dmFpbi5DbHVzdCwgc2ltLmNsdXN0Lm1lcmdlJFNhbXBsZSkpCmxvdXZhaW4uZGdlIDwtIERHRUxpc3QoY291bnRzPWxvdXZhaW4uY291bnQsIGxpYi5zaXplPWxvZyhjb2xTdW1zKGxvdXZhaW4uY291bnQpKSkKbG91dmFpbi5kZ2UgPC0gZXN0aW1hdGVEaXNwKGxvdXZhaW4uZGdlLCBsb3V2YWluLm1vZGVsKQpsb3V2YWluLmZpdCA8LSBnbG1RTEZpdChsb3V2YWluLmRnZSwgbG91dmFpbi5tb2RlbCwgcm9idXN0PVRSVUUpCmxvdXZhaW4ucmVzIDwtIGFzLmRhdGEuZnJhbWUodG9wVGFncyhnbG1RTEZUZXN0KGxvdXZhaW4uZml0LCBjb2VmPTIpLCBzb3J0LmJ5PSdub25lJywgbj1JbmYpKQp0YWJsZShsb3V2YWluLnJlcyRGRFIgPD0gMC4xKQpgYGAKCiMjIFdhbGt0cmFwCgpgYGB7ciwgd2FybmluZz1GQUxTRX0Kd2Fsa3RyYXAubW9kZWwgPC0gbW9kZWwubWF0cml4KH5Db25kaXRpb24sIGRhdGE9dGVzdC5tZXRhKQp3YWxrdHJhcC5jb3VudCA8LSBhcy5tYXRyaXgodGFibGUoc2ltLmNsdXN0Lm1lcmdlJFdhbGt0cmFwLkNsdXN0LCBzaW0uY2x1c3QubWVyZ2UkU2FtcGxlKSkKd2Fsa3RyYXAuZGdlIDwtIERHRUxpc3QoY291bnRzPXdhbGt0cmFwLmNvdW50LCBsaWIuc2l6ZT1sb2coY29sU3Vtcyh3YWxrdHJhcC5jb3VudCkpKQp3YWxrdHJhcC5kZ2UgPC0gZXN0aW1hdGVEaXNwKHdhbGt0cmFwLmRnZSwgd2Fsa3RyYXAubW9kZWwpCndhbGt0cmFwLmZpdCA8LSBnbG1RTEZpdCh3YWxrdHJhcC5kZ2UsIHdhbGt0cmFwLm1vZGVsLCByb2J1c3Q9VFJVRSkKd2Fsa3RyYXAucmVzIDwtIGFzLmRhdGEuZnJhbWUodG9wVGFncyhnbG1RTEZUZXN0KHdhbGt0cmFwLmZpdCwgY29lZj0yKSwgc29ydC5ieT0nbm9uZScsIG49SW5mKSkKdGFibGUod2Fsa3RyYXAucmVzJEZEUiA8PSAwLjEpCmBgYAoKCiMgQ29tcGFyaW5nIG1ldGhvZHMKCkFzIHN0YXRlZCBhYm92ZSwgSSB3aWxsIGFzc2VzcyB0aGUgcGVyZm9ybWFuY2Ugb2YgZWFjaCBtZXRob2RzIGJ5IGNhbGN1bGF0aW5nIHRoZSByYXRpbyBvZiBjZWxscyB0aGF0IF9zaG91bGRfIGJlIERBIGFnYWluc3QgdGhvc2UgdGhhdCBhcmUgREEgYnV0IApfc2hvdWxkIG5vdCBiZV8uCgpgYGB7ciwgd2FybmluZz1GQUxTRX0KdHJ1ZS5kYS5jZWxscyA8LSBzaW0ubWV0YSRjZWxsX2lkW3NpbS5tZXRhJGdyb3VwX2lkICVpbiUgIk0zIl0KbWlsby5kYS5jZWxscyA8LSB1bmlxdWUoc2ltLm1ldGEkY2VsbF9pZFt1bmlxdWUodW5saXN0KG5ob29kcyhzaW0ubXlsbylbbXlsby5yZXMkTmhvb2RbbXlsby5yZXMkRGlmZiAhPSAwXV0pKV0pCmN5ZGFyLmRhLmNlbGxzIDwtIHNpbS5tZXRhJGNlbGxfaWRbdW5pcXVlKHVubGlzdChjZWxsQXNzaWdubWVudHMoc2ltLmN5ZGFyKVthcy5udW1lcmljKHJvd25hbWVzKGN5ZGFyLnJlcylbY3lkYXIucmVzJFNwYXRpYWxGRFIgPD0gMC4xXSldKSldCmxvdXZhaW4uZGEuY2VsbHMgPC0gc2ltLmNsdXN0Lm1lcmdlJGNlbGxfaWRbc2ltLmNsdXN0Lm1lcmdlJExvdXZhaW4uQ2x1c3QgJWluJSByb3duYW1lcyhsb3V2YWluLnJlcylbbG91dmFpbi5yZXMkRkRSIDw9IDAuMV1dCndhbGt0cmFwLmRhLmNlbGxzIDwtIHNpbS5jbHVzdC5tZXJnZSRjZWxsX2lkW3NpbS5jbHVzdC5tZXJnZSRXYWxrdHJhcC5DbHVzdCAlaW4lIHJvd25hbWVzKHdhbGt0cmFwLnJlcylbd2Fsa3RyYXAucmVzJEZEUiA8PSAwLjFdXQpkYXNlcS5kYS5jZWxscyA8LSBzaW0ubWV0YSRjZWxsX2lkW3NpbS5kYV9yZWdpb25zJGRhLnJlZ2lvbi5sYWJlbCAhPSAwXQoKZGEuY2VsbC5kZiA8LSBkYXRhLmZyYW1lKCJNZXRob2QiPWMoIlRydXRoIiwgIk1pbG8iLCAiQ3lkYXIiLCAiTG91dmFpbiIsICJXYWxrdHJhcCIsICJEQXNlcSIpLAogICAgICAgICAgICAgICAgICAgICAgICAgIk5DZWxscyI9YyhsZW5ndGgodHJ1ZS5kYS5jZWxscyksIGxlbmd0aChtaWxvLmRhLmNlbGxzKSwgbGVuZ3RoKGN5ZGFyLmRhLmNlbGxzKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGVuZ3RoKGxvdXZhaW4uZGEuY2VsbHMpLCBsZW5ndGgod2Fsa3RyYXAuZGEuY2VsbHMpLCBsZW5ndGgoZGFzZXEuZGEuY2VsbHMpKSkKYGBgCgoKYGBge3IsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0V9CmdncGxvdChkYS5jZWxsLmRmLCBhZXMoeD1yZW9yZGVyKE1ldGhvZCwgLU5DZWxscyksIHk9TkNlbGxzKSkgKwogICAgZ2VvbV9iYXIoc3RhdD0naWRlbnRpdHknKSArCiAgICB0aGVtZV9jbGVhbigpICsKICAgIGxhYnMoeD0iREEgTWV0aG9kIiwgeT0iI0RBIGNlbGxzIikKCmdnc2F2ZSgifi9Ecm9wYm94L01pbG8vZmlndXJlcy9NZXRob2RDb21wYXJlX05EQWNlbGxzLnBkZiIsCiAgICAgICBoZWlnaHQ9My45NSwgd2lkdGg9NC4yNSwgdXNlRGluZ2JhdHM9RkFMU0UpCmBgYAoKRnJvbSB0aGlzIGl0IGRvZXMgbG9vayBsaWtlIGBNaWxvYCB3aWxsIGNhbGwgc29tZSBuZWlnaGJvdXJob29kcyB0aGF0IGFyZSBEQSBiZWNhdXNlIHRoZSByYXRpbyBvZiBjZWxscyBpbiB0aGF0IG5laWdoYm91cmhvb2QgZGVwYXJ0cyBmcm9tIDUwOjUwLiBJJ20gCm5vdCBlbnRpcmVseSBzdXJlIGhvdyBzZW5zaXRpdmUgYE1pbG9gIGlzIHRvIHRoaXMgZGVwYXJ0dXJlIGV4YWN0bHksIGJ1dCBpdCBsb29rcyBsaWtlIGl0IGNhbGxzIH44MCBvciBzbyBjZWxscyBEQSB3aGVyZSB0aGV5IHNob3VsZCBub3QgYmUuCgpgYGB7ciwgd2FybmluZz1GQUxTRX0KIyBjYWxjdWxhdGUgdHJ1ZSBEQSBjZWxscwp0cnVlLm5lZyA8LSBmYWN0b3Ioc2ltLm1ldGEkY2VsbF9pZCAlaW4lIHRydWUuZGEuY2VsbHMsIGxldmVscz1jKEZBTFNFLCBUUlVFKSkKbWlsby5uZWcgPC0gZmFjdG9yKHNpbS5tZXRhJGNlbGxfaWQgJWluJSBtaWxvLmRhLmNlbGxzLCBsZXZlbHM9YyhGQUxTRSwgVFJVRSkpCmN5ZGFyLm5lZyA8LSBmYWN0b3Ioc2ltLm1ldGEkY2VsbF9pZCAlaW4lIGN5ZGFyLmRhLmNlbGxzLCBsZXZlbHM9YyhGQUxTRSwgVFJVRSkpCmxvdXZhaW4ubmVnIDwtIGZhY3RvcihzaW0ubWV0YSRjZWxsX2lkICVpbiUgbG91dmFpbi5kYS5jZWxscywgbGV2ZWxzPWMoRkFMU0UsIFRSVUUpKQp3YWxrdHJhcC5uZWcgPC0gZmFjdG9yKHNpbS5tZXRhJGNlbGxfaWQgJWluJSB3YWxrdHJhcC5kYS5jZWxscywgbGV2ZWxzPWMoRkFMU0UsIFRSVUUpKQpkYXNlcS5uZWcgPC0gZmFjdG9yKHNpbS5tZXRhJGNlbGxfaWQgJWluJSBkYXNlcS5kYS5jZWxscywgbGV2ZWxzPWMoRkFMU0UsIFRSVUUpKQoKbWlsby5jb25mdXNlIDwtIHRhYmxlKHRydWUubmVnLCBtaWxvLm5lZykKY3lkYXIuY29uZnVzZSA8LSB0YWJsZSh0cnVlLm5lZywgY3lkYXIubmVnKQpsb3V2YWluLmNvbmZ1c2UgPC0gdGFibGUodHJ1ZS5uZWcsIGxvdXZhaW4ubmVnKQp3YWxrdHJhcC5jb25mdXNlIDwtIHRhYmxlKHRydWUubmVnLCB3YWxrdHJhcC5uZWcpCmRhc2VxLmNvbmZ1c2UgPC0gdGFibGUodHJ1ZS5uZWcsIGRhc2VxLm5lZykKCm1pbG8uY29uZnVzZS5saXN0IDwtIGxpc3QoIlRQIj1taWxvLmNvbmZ1c2VbMiwgMl0sICJGUCI9bWlsby5jb25mdXNlWzEsIDJdLCAiVE4iPW1pbG8uY29uZnVzZVsxLCAxXSwgIkZOIj1taWxvLmNvbmZ1c2VbMiwgMV0pCmN5ZGFyLmNvbmZ1c2UubGlzdCA8LSBsaXN0KCJUUCI9Y3lkYXIuY29uZnVzZVsyLCAyXSwgIkZQIj1jeWRhci5jb25mdXNlWzEsIDJdLCAiVE4iPWN5ZGFyLmNvbmZ1c2VbMSwgMV0sICJGTiI9Y3lkYXIuY29uZnVzZVsyLCAxXSkKbG91dmFpbi5jb25mdXNlLmxpc3QgPC0gbGlzdCgiVFAiPWxvdXZhaW4uY29uZnVzZVsyLCAyXSwgIkZQIj1sb3V2YWluLmNvbmZ1c2VbMSwgMl0sICJUTiI9bG91dmFpbi5jb25mdXNlWzEsIDFdLCAiRk4iPWxvdXZhaW4uY29uZnVzZVsyLCAxXSkKd2Fsa3RyYXAuY29uZnVzZS5saXN0IDwtIGxpc3QoIlRQIj13YWxrdHJhcC5jb25mdXNlWzIsIDJdLCAiRlAiPXdhbGt0cmFwLmNvbmZ1c2VbMSwgMl0sICJUTiI9d2Fsa3RyYXAuY29uZnVzZVsxLCAxXSwgIkZOIj13YWxrdHJhcC5jb25mdXNlWzIsIDFdKQpkYXNlcS5jb25mdXNlLmxpc3QgPC0gbGlzdCgiVFAiPWRhc2VxLmNvbmZ1c2VbMiwgMl0sICJGUCI9ZGFzZXEuY29uZnVzZVsxLCAyXSwgIlROIj1kYXNlcS5jb25mdXNlWzEsIDFdLCAiRk4iPWRhc2VxLmNvbmZ1c2VbMiwgMV0pCmBgYAoKCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQptaWxvLnBwdiA8LSBtaWxvLmNvbmZ1c2UubGlzdCRUUC8obWlsby5jb25mdXNlLmxpc3QkVFAgKyBtaWxvLmNvbmZ1c2UubGlzdCRGUCkKY3lkYXIucHB2IDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRUUC8oY3lkYXIuY29uZnVzZS5saXN0JFRQICsgY3lkYXIuY29uZnVzZS5saXN0JEZQKQpsb3V2YWluLnBwdiA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRUUC8obG91dmFpbi5jb25mdXNlLmxpc3QkVFAgKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGUCkKd2Fsa3RyYXAucHB2IDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRUUC8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFRQICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZQKQpkYXNlcS5wcHYgPC0gZGFzZXEuY29uZnVzZS5saXN0JFRQLyhkYXNlcS5jb25mdXNlLmxpc3QkVFAgKyBkYXNlcS5jb25mdXNlLmxpc3QkRlApCgptaWxvLmZkciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRGUC8obWlsby5jb25mdXNlLmxpc3QkVFAgKyBtaWxvLmNvbmZ1c2UubGlzdCRGUCkKY3lkYXIuZmRyIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRGUC8oY3lkYXIuY29uZnVzZS5saXN0JFRQICsgY3lkYXIuY29uZnVzZS5saXN0JEZQKQpsb3V2YWluLmZkciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRGUC8obG91dmFpbi5jb25mdXNlLmxpc3QkVFAgKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGUCkKd2Fsa3RyYXAuZmRyIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRGUC8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFRQICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZQKQpkYXNlcS5mZHIgPC0gZGFzZXEuY29uZnVzZS5saXN0JEZQLyhkYXNlcS5jb25mdXNlLmxpc3QkVFAgKyBkYXNlcS5jb25mdXNlLmxpc3QkRlApCgptaWxvLmZuciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRGTi8obWlsby5jb25mdXNlLmxpc3QkVFAgKyBtaWxvLmNvbmZ1c2UubGlzdCRGTikKY3lkYXIuZm5yIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRGTi8oY3lkYXIuY29uZnVzZS5saXN0JFRQICsgY3lkYXIuY29uZnVzZS5saXN0JEZOKQpsb3V2YWluLmZuciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRGTi8obG91dmFpbi5jb25mdXNlLmxpc3QkVFAgKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGTikKd2Fsa3RyYXAuZm5yIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRGTi8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFRQICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZOKQpkYXNlcS5mbnIgPC0gZGFzZXEuY29uZnVzZS5saXN0JEZOLyhkYXNlcS5jb25mdXNlLmxpc3QkVFAgKyBkYXNlcS5jb25mdXNlLmxpc3QkRk4pCgptaWxvLmZwciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRGUC8obWlsby5jb25mdXNlLmxpc3QkRlAgKyBtaWxvLmNvbmZ1c2UubGlzdCRUTikKY3lkYXIuZnByIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRGUC8oY3lkYXIuY29uZnVzZS5saXN0JEZQICsgY3lkYXIuY29uZnVzZS5saXN0JFROKQpsb3V2YWluLmZwciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRGUC8obG91dmFpbi5jb25mdXNlLmxpc3QkRlAgKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRUTikKd2Fsa3RyYXAuZnByIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRGUC8od2Fsa3RyYXAuY29uZnVzZS5saXN0JEZQICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JFROKQpkYXNlcS5mcHIgPC0gZGFzZXEuY29uZnVzZS5saXN0JEZQLyhkYXNlcS5jb25mdXNlLmxpc3QkRlAgKyBkYXNlcS5jb25mdXNlLmxpc3QkVE4pCgptaWxvLmZvciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRGTi8obWlsby5jb25mdXNlLmxpc3QkRk4gKyBtaWxvLmNvbmZ1c2UubGlzdCRUTikKY3lkYXIuZm9yIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRGTi8oY3lkYXIuY29uZnVzZS5saXN0JEZOICsgY3lkYXIuY29uZnVzZS5saXN0JFROKQpsb3V2YWluLmZvciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRGTi8obG91dmFpbi5jb25mdXNlLmxpc3QkRk4gKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRUTikKd2Fsa3RyYXAuZm9yIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRGTi8od2Fsa3RyYXAuY29uZnVzZS5saXN0JEZOICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JFROKQpkYXNlcS5mb3IgPC0gZGFzZXEuY29uZnVzZS5saXN0JEZOLyhkYXNlcS5jb25mdXNlLmxpc3QkRk4gKyBkYXNlcS5jb25mdXNlLmxpc3QkVE4pCgptaWxvLnRwciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRUUC8obWlsby5jb25mdXNlLmxpc3QkVFAgKyBtaWxvLmNvbmZ1c2UubGlzdCRGTikKY3lkYXIudHByIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRUUC8oY3lkYXIuY29uZnVzZS5saXN0JFRQICsgY3lkYXIuY29uZnVzZS5saXN0JEZOKQpsb3V2YWluLnRwciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRUUC8obG91dmFpbi5jb25mdXNlLmxpc3QkVFAgKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGTikKd2Fsa3RyYXAudHByIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRUUC8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFRQICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZOKQpkYXNlcS50cHIgPC0gZGFzZXEuY29uZnVzZS5saXN0JFRQLyhkYXNlcS5jb25mdXNlLmxpc3QkVFAgKyBkYXNlcS5jb25mdXNlLmxpc3QkRk4pCgptaWxvLnRuciA8LSBtaWxvLmNvbmZ1c2UubGlzdCRUTi8obWlsby5jb25mdXNlLmxpc3QkVE4gKyBtaWxvLmNvbmZ1c2UubGlzdCRGUCkKY3lkYXIudG5yIDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRUTi8oY3lkYXIuY29uZnVzZS5saXN0JFROICsgY3lkYXIuY29uZnVzZS5saXN0JEZQKQpsb3V2YWluLnRuciA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRUTi8obG91dmFpbi5jb25mdXNlLmxpc3QkVE4gKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGUCkKd2Fsa3RyYXAudG5yIDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRUTi8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFROICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZQKQpkYXNlcS50bnIgPC0gZGFzZXEuY29uZnVzZS5saXN0JFROLyhkYXNlcS5jb25mdXNlLmxpc3QkVE4gKyBkYXNlcS5jb25mdXNlLmxpc3QkRlApCgptaWxvLm5wdiA8LSBtaWxvLmNvbmZ1c2UubGlzdCRUTi8obWlsby5jb25mdXNlLmxpc3QkVE4gKyBtaWxvLmNvbmZ1c2UubGlzdCRGTikKY3lkYXIubnB2IDwtIGN5ZGFyLmNvbmZ1c2UubGlzdCRUTi8oY3lkYXIuY29uZnVzZS5saXN0JFROICsgY3lkYXIuY29uZnVzZS5saXN0JEZOKQpsb3V2YWluLm5wdiA8LSBsb3V2YWluLmNvbmZ1c2UubGlzdCRUTi8obG91dmFpbi5jb25mdXNlLmxpc3QkVE4gKyBsb3V2YWluLmNvbmZ1c2UubGlzdCRGTikKd2Fsa3RyYXAubnB2IDwtIHdhbGt0cmFwLmNvbmZ1c2UubGlzdCRUTi8od2Fsa3RyYXAuY29uZnVzZS5saXN0JFROICsgd2Fsa3RyYXAuY29uZnVzZS5saXN0JEZOKQpkYXNlcS5ucHYgPC0gZGFzZXEuY29uZnVzZS5saXN0JFROLyhkYXNlcS5jb25mdXNlLmxpc3QkVE4gKyBkYXNlcS5jb25mdXNlLmxpc3QkRk4pCgpkYS5mZHIuZGYgPC0gZGF0YS5mcmFtZSgiTWV0aG9kIj1jKCJNaWxvIiwgIkN5ZGFyIiwgIkxvdXZhaW4iLCAiV2Fsa3RyYXAiLCAiREFzZXEiKSwKICAgICAgICAgICAgICAgICAgICAgICAgIlBQViI9YyhtaWxvLnBwdiwgY3lkYXIucHB2LCBsb3V2YWluLnBwdiwgd2Fsa3RyYXAucHB2LCBkYXNlcS5wcHYpLAogICAgICAgICAgICAgICAgICAgICAgICAiTlBWIj1jKG1pbG8ubnB2LCBjeWRhci5ucHYsIGxvdXZhaW4ubnB2LCB3YWxrdHJhcC5ucHYsIGRhc2VxLm5wdiksCiAgICAgICAgICAgICAgICAgICAgICAgICJGRFIiPWMobWlsby5mZHIsIGN5ZGFyLmZkciwgbG91dmFpbi5mZHIsIHdhbGt0cmFwLmZkciwgZGFzZXEuZmRyKSwKICAgICAgICAgICAgICAgICAgICAgICAgIkZQUiI9YyhtaWxvLmZwciwgY3lkYXIuZnByLCBsb3V2YWluLmZwciwgd2Fsa3RyYXAuZnByLCBkYXNlcS5mcHIpLAogICAgICAgICAgICAgICAgICAgICAgICAiRk5SIj1jKG1pbG8uZm5yLCBjeWRhci5mbnIsIGxvdXZhaW4uZm5yLCB3YWxrdHJhcC5mbnIsIGRhc2VxLmZuciksCiAgICAgICAgICAgICAgICAgICAgICAgICJGT1IiPWMobWlsby5mb3IsIGN5ZGFyLmZvciwgbG91dmFpbi5mb3IsIHdhbGt0cmFwLmZvciwgZGFzZXEuZm9yKSwKICAgICAgICAgICAgICAgICAgICAgICAgIlRQUiI9YyhtaWxvLnRwciwgY3lkYXIudHByLCBsb3V2YWluLnRwciwgd2Fsa3RyYXAudHByLCBkYXNlcS50cHIpLAogICAgICAgICAgICAgICAgICAgICAgICAiVE5SIj1jKG1pbG8udG5yLCBjeWRhci50bnIsIGxvdXZhaW4udG5yLCB3YWxrdHJhcC50bnIsIGRhc2VxLnRucikpCmRhLmZkci5kZiRNQ0MgPC0gc3FydChkYS5mZHIuZGYkUFBWICogZGEuZmRyLmRmJFRQUiAqIGRhLmZkci5kZiRUTlIgKiBkYS5mZHIuZGYkTlBWKSAtIAogICAgc3FydChkYS5mZHIuZGYkRkRSICogZGEuZmRyLmRmJEZOUiAqIGRhLmZkci5kZiRGUFIgKiBkYS5mZHIuZGYkRk9SKQpkYS5mZHIuZGYkRjEgPC0gMiooKGRhLmZkci5kZiRQUFYgKiBkYS5mZHIuZGYkVFBSKS8oZGEuZmRyLmRmJFBQViArIGRhLmZkci5kZiRUUFIpKQpkYS5mZHIuZGYkUG93ZXIgPC0gMSAtIGRhLmZkci5kZiRGTlIKYGBgCgpJJ3ZlIGNvbXB1dGUgdGhlIGNvbmZ1c2lvbiBtYXRyaXggZm9yIGVhY2ggbWV0aG9kLCBmcm9tIHdoaWNoIEkgaGF2ZSBjYWxjdWxhdGVkIHRoZSBwcmVjaXNpb24gKHBvc2l0aXZlIHByZWRpY3RpdmUgdmFsdWUpLCByZWNhbGwgKFRQUiksIEZEUiBhbmQgCk1hdHRoZXdzIGNvcnJlbGF0aW9uIGNvZWZmaWNpZW50IChNQ0MpLgoKYGBge3IsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0UsIGZpZy5oZWlnaHQ9My45NSwgZmlnLndpZHRoPTQuMjV9CmdncGxvdChkYS5mZHIuZGYsIGFlcyh4PXJlb3JkZXIoTWV0aG9kLCAtUFBWKSwgeT1QUFYpKSArCiAgICBnZW9tX2JhcihzdGF0PSdpZGVudGl0eScpICsKICAgIHRoZW1lX2NsZWFuKCkgKwogICAgbGFicyh4PSJEQSBNZXRob2QiLCB5PSJQUFYiKQoKZ2dzYXZlKCJ+L0Ryb3Bib3gvTWlsby9maWd1cmVzL01ldGhvZENvbXBhcmVfUFBWLnBkZiIsCiAgICAgICBoZWlnaHQ9My45NSwgd2lkdGg9NC4yNSwgdXNlRGluZ2JhdHM9RkFMU0UpCmBgYAoKCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFLCBmaWcuaGVpZ2h0PTMuOTUsIGZpZy53aWR0aD00LjI1fQpnZ3Bsb3QoZGEuZmRyLmRmLCBhZXMoeD1yZW9yZGVyKE1ldGhvZCwgLUZEUiksIHk9RkRSKSkgKwogICAgZ2VvbV9iYXIoc3RhdD0naWRlbnRpdHknKSArCiAgICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9MC4xLCBsdHk9MiwgY29sPSdyZWQnKSArCiAgICB0aGVtZV9jbGVhbigpICsKICAgIGxhYnMoeD0iREEgTWV0aG9kIiwgeT0iQ2VsbC13aXNlIEZEUiIpCgpnZ3NhdmUoIn4vRHJvcGJveC9NaWxvL2ZpZ3VyZXMvTWV0aG9kQ29tcGFyZV9GRFIucGRmIiwKICAgICAgIGhlaWdodD0zLjk1LCB3aWR0aD00LjI1LCB1c2VEaW5nYmF0cz1GQUxTRSkKYGBgCgoKYGBge3IsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0UsIGZpZy5oZWlnaHQ9My45NSwgZmlnLndpZHRoPTQuMjV9CmdncGxvdChkYS5mZHIuZGYsIGFlcyh4PXJlb3JkZXIoTWV0aG9kLCAtVFBSKSwgeT1UUFIpKSArCiAgICBnZW9tX2JhcihzdGF0PSdpZGVudGl0eScpICsKICAgIHRoZW1lX2NsZWFuKCkgKwogICAgbGFicyh4PSJEQSBNZXRob2QiLCB5PSJSZWNhbGwiKQoKZ2dzYXZlKCJ+L0Ryb3Bib3gvTWlsby9maWd1cmVzL01ldGhvZENvbXBhcmVfUmVjYWxsLnBkZiIsCiAgICAgICBoZWlnaHQ9My45NSwgd2lkdGg9NC4yNSwgdXNlRGluZ2JhdHM9RkFMU0UpCmBgYAoKCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFLCBmaWcuaGVpZ2h0PTMuOTUsIGZpZy53aWR0aD00LjI1fQpnZ3Bsb3QoZGEuZmRyLmRmLCBhZXMoeD1yZW9yZGVyKE1ldGhvZCwgLU1DQyksIHk9TUNDKSkgKwogICAgZ2VvbV9iYXIoc3RhdD0naWRlbnRpdHknKSArCiAgICB0aGVtZV9jb3dwbG90KCkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTIwKSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MjIpKSArCiAgICBsYWJzKHg9Ik1ldGhvZCIsIHk9Ik1DQyIpCgpnZ3NhdmUoIn4vRHJvcGJveC9NaWxvL2ZpZ3VyZXMvTWV0aG9kQ29tcGFyZV9NQ0MucGRmIiwKICAgICAgIGhlaWdodD0yLjk1LCB3aWR0aD02Ljk1LCB1c2VEaW5nYmF0cz1GQUxTRSkKYGBgCgpXaGF0IGRvZXMgdGhlIHBvd2VyIGxvb2sgbGlrZT8KCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBmaWcuaGVpZ2h0PTMuOTUsIGZpZy53aWR0aD00LjI1fQpnZ3Bsb3QoZGEuZmRyLmRmLCBhZXMoeD1yZW9yZGVyKE1ldGhvZCwgLVBvd2VyKSwgeT1Qb3dlcikpICsKICAgIGdlb21fYmFyKHN0YXQ9J2lkZW50aXR5JykgKwogICAgdGhlbWVfY2xlYW4oKSArCiAgICBsYWJzKHg9IkRBIE1ldGhvZCIsIHk9IlBvd2VyIikKCmdnc2F2ZSgifi9Ecm9wYm94L01pbG8vZmlndXJlcy9NZXRob2RDb21wYXJlX1Bvd2VyLnBkZiIsCiAgICAgICBoZWlnaHQ9My45NSwgd2lkdGg9NC4yNSwgdXNlRGluZ2JhdHM9RkFMU0UpCmBgYAoKTGV0J3MgdmlzdWFsaXNlIHRoZXNlIGluIGEgc2luZ2xlIHRhYmxlL21hdHJpeC4gSSBoYXZlIHRvIHNwbGl0IHRoaXMgaW50byB0aGUgbWVhc3VyZXMgd2hlcmUgaGlnaGVyIGlzIGJldHRlciBhbmQgCmxvd2VyIGlzIGJldHRlci4KCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQojIGNhbGN1bGF0ZSB0aGUgcmFuayBhbG9uZyBlYWNoIGNvbHVtbgpkYS5uZWdyYW5rLmRmIDwtIGFzLmRhdGEuZnJhbWUoYXBwbHkoZGEuZmRyLmRmWywgYygiUG93ZXIiLCAiRjEiLCAiVE5SIiwgIlRQUiIsICJOUFYiLCAiUFBWIildLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMiwgRlVOPWZ1bmN0aW9uKFgpIHJhbmsoLVgpKSkKZGEubmVncmFuay5kZiRNZXRob2QgPC0gZGEuZmRyLmRmJE1ldGhvZApkYS5uZWdyYW5rLm1lbHQgPC0gbWVsdChkYS5uZWdyYW5rLmRmLCBpZC52YXJzPWMoIk1ldGhvZCIpKQpkYS5uZWdyYW5rLm1lbHQkdmFsdWUgPC0gb3JkZXJlZChkYS5uZWdyYW5rLm1lbHQkdmFsdWUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxldmVscz1jKDE6NSkpCgpyYW5rLmNvbHMgPC0gY29sb3JSYW1wUGFsZXR0ZShwYWxfZnV0dXJhbWEoKSgzKSkoNSkKbmFtZXMocmFuay5jb2xzKSA8LSBjKDE6NSkKCmdncGxvdChkYS5uZWdyYW5rLm1lbHQsIAogICAgICAgYWVzKHg9TWV0aG9kLCB5PXZhcmlhYmxlLCBmaWxsPXZhbHVlKSkgKwogICAgZ2VvbV90aWxlKCkgKwogICAgdGhlbWVfY293cGxvdCgpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcz1yYW5rLmNvbHMpICsKICAgIGxhYnMoeD0iTWV0aG9kIiwgeT0iTWVhc3VyZSIpICsKICAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0yMiksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MjApKSArCiAgICBndWlkZXMoZmlsbD1ndWlkZV9sZWdlbmQodGl0bGU9IlJhbmsiKSkKCmdnc2F2ZSgifi9Ecm9wYm94L01pbG8vZmlndXJlcy9NZXRob2RDb21wYXJlX1Bvc1JhbmtfdGFibGUucGRmIiwKICAgICAgIGhlaWdodD0zLjk1LCB3aWR0aD02LjI1LCB1c2VEaW5nYmF0cz1GQUxTRSkKYGBgCgpgYGB7ciwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KIyBjYWxjdWxhdGUgdGhlIHJhbmsgYWxvbmcgZWFjaCBjb2x1bW4KZGEucG9zcmFuay5kZiA8LSBhcy5kYXRhLmZyYW1lKGFwcGx5KGRhLmZkci5kZlssIGMoIkZEUiIsICJGUFIiLCAiRk5SIiwgIkZPUiIpXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIDIsIEZVTj1mdW5jdGlvbihYKSByYW5rKFgpKSkKZGEucG9zcmFuay5kZiRNZXRob2QgPC0gZGEuZmRyLmRmJE1ldGhvZApkYS5wb3NyYW5rLm1lbHQgPC0gbWVsdChkYS5wb3NyYW5rLmRmLCBpZC52YXJzPWMoIk1ldGhvZCIpKQpkYS5wb3NyYW5rLm1lbHQkdmFsdWUgPC0gb3JkZXJlZChkYS5wb3NyYW5rLm1lbHQkdmFsdWUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxldmVscz1jKDE6NSkpCgpyYW5rLmNvbHMgPC0gY29sb3JSYW1wUGFsZXR0ZShwYWxfZnV0dXJhbWEoKSgzKSkoNSkKbmFtZXMocmFuay5jb2xzKSA8LSBjKDE6NSkKCmdncGxvdChkYS5wb3NyYW5rLm1lbHQsIAogICAgICAgYWVzKHg9TWV0aG9kLCB5PXZhcmlhYmxlLCBmaWxsPXZhbHVlKSkgKwogICAgZ2VvbV90aWxlKCkgKwogICAgdGhlbWVfY293cGxvdCgpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcz1yYW5rLmNvbHMpICsKICAgIGxhYnMoeD0iTWV0aG9kIiwgeT0iTWVhc3VyZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTIyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0yMCkpICsKICAgIGd1aWRlcyhmaWxsPWd1aWRlX2xlZ2VuZCh0aXRsZT0iUmFuayIpKQoKZ2dzYXZlKCJ+L0Ryb3Bib3gvTWlsby9maWd1cmVzL01ldGhvZENvbXBhcmVfTmVnUmFua190YWJsZS5wZGYiLAogICAgICAgaGVpZ2h0PTMuOTUsIHdpZHRoPTYuMjUsIHVzZURpbmdiYXRzPUZBTFNFKQpgYGAKCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQojIGNhbGN1bGF0ZSB0aGUgcmFuayBhbG9uZyBlYWNoIGNvbHVtbgphbGxyYW5rLm1lbHQgPC0gZG8uY2FsbChyYmluZC5kYXRhLmZyYW1lLCBsaXN0KCJwb3N0Ij1kYS5wb3NyYW5rLm1lbHQsICJuZWciPWRhLm5lZ3JhbmsubWVsdCkpCgpyYW5rLmNvbHMgPC0gY29sb3JSYW1wUGFsZXR0ZShwYWxfZnV0dXJhbWEoKSgzKSkoNSkKbmFtZXMocmFuay5jb2xzKSA8LSBjKDE6NSkKCmdncGxvdChhbGxyYW5rLm1lbHQsIAogICAgICAgYWVzKHg9TWV0aG9kLCB5PXZhcmlhYmxlLCBmaWxsPXZhbHVlKSkgKwogICAgZ2VvbV90aWxlKCkgKwogICAgdGhlbWVfY293cGxvdCgpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcz1yYW5rLmNvbHMpICsKICAgIGxhYnMoeD0iTWV0aG9kIiwgeT0iTWVhc3VyZSIpICsKICAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0yMiksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MjApKSArCiAgICBndWlkZXMoZmlsbD1ndWlkZV9sZWdlbmQodGl0bGU9IlJhbmsiKSkKCmdnc2F2ZSgifi9Ecm9wYm94L01pbG8vZmlndXJlcy9NZXRob2RDb21wYXJlX0FsbFJhbmtfdGFibGUucGRmIiwKICAgICAgIGhlaWdodD00LjE1LCB3aWR0aD02Ljk1LCB1c2VEaW5nYmF0cz1GQUxTRSkKYGBgCgoK